package jarvis.fs.analyzer;

import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
/**
 * 
 * Function: Remove bad words whose length is 1 or be a float
 * @author Jarvis.Guo
 *
 */
public class BadWordsFilter extends TokenFilter {

	
	private Pattern pattern;


	
	protected BadWordsFilter(TokenStream input) {
		super(input);
		pattern = Pattern.compile("[\\.\\d]+");
	}

	public Token next() throws IOException {
		Token result = null;
		while((result = input.next()) != null)
		{
			String text = result.termText();
			if(text.length()==1) continue;//ignore length=1's word
			Matcher m = pattern.matcher(text);
			if(m.matches()) continue;//is float
			return result;
		}
		return null;
	}

}
